package cn.demo.gecco.starter;

import com.geccocrawler.gecco.GeccoEngine;
import com.geccocrawler.gecco.request.HttpGetRequest;


import java.util.HashMap;
import java.util.Map;

/**
 * @ClassName:
 * @PackageName: cn.demo.gecco.starter
 * @author: youjp
 * @create: 2019-11-04 08:39
 * @description: 爬虫启动类
 * @Version: 1.0
 */
public class StartClass {

    public static void main(String[] args) {

        System.out.println("=========start===============");
      //  HttpGetRequest startUrl = new HttpGetRequest("http://localhost:8081/a1.html");
  HttpGetRequest startUrl = new HttpGetRequest("https://shanghai.anjuke.com/sale/a8-p1/#filtersort");
        //HttpGetRequest startUrl= new HttpGetRequest("https://zhaotong.anjuke.com/sale/rd1/?kw=&from=sugg");
   // HttpGetRequest startUrl= new HttpGetRequest("https://pixabay.com/zh/images/search/?pagi=1");
 //  HttpGetRequest startUrl= new HttpGetRequest("https://www.veer.com/photo/");
        startUrl.setCharset("utf-8");
        startUrl.setHeaders(setHeaders());

        GeccoEngine.create()    //Gecco搜索包路径
                .classpath("cn.demo.gecco")
                .start(startUrl)
                .thread(1)    //开启几个爬虫线程
                .interval(5000)     //单个爬虫每次抓取完一个请求后的间隔时间
                .run();
    }

    /**
     * @return java.util.Map<java.lang.String               ,               java.lang.String>
     * @Param []
     * @Author youjp
     * @Description //TODO 设置请求头
     * @throw
     **/
    public static Map<String, String> setHeaders() {
        Map<String, String> setHeaders = new HashMap<>();
        setHeaders.put("Accept", "text/html,application/xhtml+xml," +
                "application/xml;q=0.9,image/webp,*/*;q=0.8");

        setHeaders.put("cookie","__cfduid=d72b498105bac6216ad85b20c5353d47d1595397878; _ga=GA1.2.445552814.1595397879; _gid=GA1.2.910806637.1595397879");
        setHeaders.put(":authority","cdn.pixabay.com");
        setHeaders.put("Accept-Encoding", "gzip, deflate, sdch, br");
        setHeaders.put("Accept-Language", "zh-CN,zh;q=0.8");
        setHeaders.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.104 Safari/537.36");
        return setHeaders;
    }
}
